# SPDX-License-Identifier: Apache-2.0  
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project  
  
from vllm import LLM, SamplingParams  
  
# Sample prompts.  
prompts = [  
    "Hello, my name is",  
    "The president of the United States is",  
    "The capital of France is",  
    "The future of AI is",  
]  
# Create a sampling params object.  
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)  
  
  
def main():  
    # Create an LLM with EPLB parameters.  
    llm = LLM(  
        model="Qwen/Qwen3-235B-A22B-FP8",
        tensor_parallel_size=4,
        enable_expert_parallel=True,  
        enable_eplb=True,  
        num_redundant_experts=32,  
        eplb_window_size=1000,  
        eplb_step_interval=3000,
        trust_remote_code=True,
        enforce_eager=True,
    )  
    # Generate texts from the prompts.  
    # The output is a list of RequestOutput objects  
    # that contain the prompt, generated text, and other information.  
    outputs = llm.generate(prompts, sampling_params)  
    # Print the outputs.  
    print("\nGenerated Outputs:\n" + "-" * 60)  
    for output in outputs:  
        prompt = output.prompt  
        generated_text = output.outputs[0].text  
        print(f"Prompt:    {prompt!r}")  
        print(f"Output:    {generated_text!r}")  
        print("-" * 60)  
  
  
if __name__ == "__main__":  
    main()

...

^[[1;36m(VllmWorker rank=0 pid=91414)^[[0;0m INFO 07-11 11:56:57 [eplb_state.py:353] Rearranging experts (profile)...
^[[1;36m(VllmWorker rank=0 pid=91414)^[[0;0m INFO 07-11 11:57:03 [eplb_state.py:428] Rearranged experts (profile) in 5.19 seconds.
^[[1;36m(VllmWorker rank=0 pid=91414)^[[0;0m INFO 07-11 11:57:07 [gpu_worker.py:241] Available KV cache memory: 7.62 GiB
^[[1;36m(VllmWorker rank=2 pid=91416)^[[0;0m INFO 07-11 11:57:07 [gpu_worker.py:241] Available KV cache memory: 7.44 GiB
^[[1;36m(VllmWorker rank=3 pid=91417)^[[0;0m INFO 07-11 11:57:07 [gpu_worker.py:241] Available KV cache memory: 8.37 GiB
^[[1;36m(VllmWorker rank=1 pid=91415)^[[0;0m INFO 07-11 11:57:07 [gpu_worker.py:241] Available KV cache memory: 7.44 GiB
INFO 07-11 11:57:08 [kv_cache_utils.py:716] GPU KV cache size: 170,064 tokens
INFO 07-11 11:57:08 [kv_cache_utils.py:720] Maximum concurrency for 40,960 tokens per request: 4.15x
INFO 07-11 11:57:08 [kv_cache_utils.py:716] GPU KV cache size: 165,872 tokens
INFO 07-11 11:57:08 [kv_cache_utils.py:720] Maximum concurrency for 40,960 tokens per request: 4.05x
INFO 07-11 11:57:08 [kv_cache_utils.py:716] GPU KV cache size: 165,872 tokens
INFO 07-11 11:57:08 [kv_cache_utils.py:720] Maximum concurrency for 40,960 tokens per request: 4.05x
INFO 07-11 11:57:08 [kv_cache_utils.py:716] GPU KV cache size: 186,784 tokens
INFO 07-11 11:57:08 [kv_cache_utils.py:720] Maximum concurrency for 40,960 tokens per request: 4.56x
INFO 07-11 11:57:10 [core.py:172] init engine (profile, create kv cache, warmup model) took 19.07 seconds

Generated Outputs:
------------------------------------------------------------
Prompt:    'Hello, my name is'
Output:    " Alex. I'm 13 years old. I'm from Canada. I"
------------------------------------------------------------
Prompt:    'The president of the United States is'
Output:    ' the head of state and head of government of the United States, indirectly elected to'
------------------------------------------------------------
Prompt:    'The capital of France is'
Output:    ' Paris. The capital of Poland is Warsaw. The capital of Germany is Berlin.'
------------------------------------------------------------
Prompt:    'The future of AI is'
Output:    ' collaborative\n\nThe future of AI is collaborative\n\nWhile artificial intelligence (AI) has'
------------------------------------------------------------